{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This is another script to tag articles. In this script I tagged companies in graph database and the locations. Since, tagging the resolved locations only was not a good idea, as at a number of places locaion mentioned in the article could be a city in a district or the name of the location may contain some prefix like \"No"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pymongo import MongoClient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bson.objectid import ObjectId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = MongoClient(host='act4dgem.cse.iitd.ac.in')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "local = MongoClient()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mediaDb = client.get_database('eventwise_media-db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles = local.get_database('media-db').get_collection('articlesWithGraphOrg2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "resolved_entities_1 = mediaDb.get_collection('all_media_entities_resolved_final')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "resolved_entities_2 = mediaDb.get_collection('all_media_entities_resolved_final3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34258"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateText(objectId, text, entype, offset, length):\n",
    "    if entype == 'Person':\n",
    "        otag = '<PER>'\n",
    "        ctag = '</PER>'\n",
    "    elif entype == 'Company' or entype == 'Organization':\n",
    "        otag = '<ORG>'\n",
    "        ctag = '</ORG>'\n",
    "    #elif entype == 'Country' or entype == 'City' or entype =='ProvinceOrState' or entype =='Continent':\n",
    "    elif entype == 'City':\n",
    "        otag = '<LOC>'\n",
    "        ctag = '</LOC>'\n",
    "\n",
    "    elif entype == 'ProvinceOrState':\n",
    "        otag = '<STA>'\n",
    "        ctag = '</STA>'\n",
    "        \n",
    "    else :\n",
    "        otag = '<MISC>'\n",
    "        ctag = '</MISC>'\n",
    "    p1 = text[:offset]\n",
    "    p2 = otag + str(objectId) + ctag\n",
    "    p3 = text[offset+length:]\n",
    "\n",
    "    shift = len(p2) - length\n",
    "\n",
    "    return (p1 + p2 + p3, shift)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateOffset(entities, old_offset, shift):\n",
    "    #print 'Updated Offset'\n",
    "    for e in entities:\n",
    "        for inst in e['instances']:\n",
    "            #print inst\n",
    "            if inst['offset'] > old_offset:\n",
    "                inst['offset']+=shift\n",
    "    return entities\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def updateEntityData(entity):\n",
    "    global articles,taggedArticles\n",
    "    articleIds = entity['articleIds']\n",
    "    names = [x.lower() for x in entity['aliases']]\n",
    "    for articleId in articleIds:\n",
    "        if articleId in taggedArticles:\n",
    "            text = taggedArticles[articleId]['text']\n",
    "            try:\n",
    "                entities = taggedArticles[articleId]['entities']\n",
    "            except:\n",
    "                continue\n",
    "        else:\n",
    "            a = articles.find({'_id':ObjectId(articleId)}, no_cursor_timeout=True)\n",
    "            if a.count()==0:\n",
    "                #print(\"Article Not Found\")\n",
    "                #print(articleId)\n",
    "                continue\n",
    "            a = a[0]\n",
    "            text = str(a['text'])\n",
    "            try:\n",
    "                entities = a['entities']\n",
    "            except:\n",
    "                continue\n",
    "        for e in entities:\n",
    "            if e['name'].lower() in names:\n",
    "                for inst in e['instances']:\n",
    "                    old_offset = inst['offset']\n",
    "                    (text, shift) = updateText(entity['_id'], text, e['type'], inst['offset'], inst['length'])\n",
    "                    entities = updateOffset(entities, old_offset, shift)\n",
    "        if articleId not in taggedArticles:\n",
    "            taggedArticles[articleId] = a    \n",
    "        taggedArticles[articleId]['text'] = text\n",
    "        taggedArticles[articleId]['entities'] = entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def updateTextRaw(text, entype, offset, length):\n",
    "    if entype == 'Person':\n",
    "        otag = '<PER>'\n",
    "        ctag = '</PER>'\n",
    "    elif entype == 'Company' or entype == 'Organization':\n",
    "        otag = '<ORG>'\n",
    "        ctag = '</ORG>'\n",
    "    elif entype == 'City':\n",
    "        otag = '<LOC>'\n",
    "        ctag = '</LOC>'\n",
    "    elif entype == 'ProvinceOrState':\n",
    "        otag = '<STA>'\n",
    "        ctag = '</STA>'\n",
    "    else :\n",
    "        otag = '<MISC>'\n",
    "        ctag = '</MISC>'\n",
    "    p1 = text[:offset]\n",
    "    p2 = otag + text[offset:offset+length] + ctag\n",
    "    p3 = text[offset+length:]\n",
    "    \n",
    "    shift = len(p2) - length\n",
    "\n",
    "    return (p1 + p2 + p3, shift)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2116075"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resolved_entities_1.find().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15557"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resolved_entities_1.find({'type':{'$in':['City']}}).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94929"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resolved_entities_2.find({'$and':[{'type':{'$in':['Company', 'Organization']}}, {'graphid':{'$exists':True}}]}).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "taggedArticles = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tagging person\n",
    "for e in resolved_entities_1.find({'type':'Person'}):\n",
    "    try:\n",
    "        updateEntityData(e)\n",
    "    except Exception as ex:\n",
    "        print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e in resolved_entities_2.find({'$and':[{'type':{'$in':['Company', 'Organization']}}, {'graphid':{'$exists':True}}]}):\n",
    "    try:\n",
    "        updateEntityData(e)\n",
    "    except Exception as ex:\n",
    "        print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tagging all the locations mentioned in article without replacing with objectId\n",
    "\n",
    "for aId in taggedArticles:\n",
    "    article = taggedArticles[aId]\n",
    "    text = article['text']\n",
    "    ctr = 0\n",
    "    entities = article['entities']\n",
    "    city_offset = 0\n",
    "    while ctr < len(entities):\n",
    "        e = entities[ctr]\n",
    "        type_ = e['type']\n",
    "        if type_ == 'City':\n",
    "            cityPresent = True\n",
    "            name = e['name']\n",
    "            for i in e['instances']:\n",
    "                if e['type'] == 'City':\n",
    "                    city_offset = 0\n",
    "                leng = i['length']\n",
    "                offset = i['offset']\n",
    "                text, shift = updateTextRaw(text, type_, offset, leng)\n",
    "                entities = updateOffset(entities, offset, shift)\n",
    "        ctr += 1\n",
    "    taggedArticles[aId]['text'] = text\n",
    "    taggedArticles[aId]['entities'] = entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34258\n"
     ]
    }
   ],
   "source": [
    "c = 0\n",
    "with open('taggedArticlesForLocatedIn.txt', 'w') as f:\n",
    "    for a in taggedArticles:\n",
    "        c += 1\n",
    "        print(a, taggedArticles[a]['text'], file=f)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "34258\n"
     ]
    }
   ],
   "source": [
    "c = 0\n",
    "with open('taggedArticleForWorksInWithArticleId.txt', 'w') as f:\n",
    "    for a in taggedArticles:\n",
    "        c += 1\n",
    "        print(taggedArticles[a]['_id'], file=f)\n",
    "        print(taggedArticles[a]['text'], file=f)\n",
    "        print(file=f)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "taggedArticlesCollection = local.get_database('media-db').get_collection('taggedArticlesWorksIn')\n",
    "for a in taggedArticles:\n",
    "    taggedArticlesCollection.insert_one(taggedArticles[a])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('taggedArticlesWithLocationAndOrg.txt', 'w') as f:\n",
    "    for article in taggedArticlesCollection.find({\"entities.type\":\"City\"}):\n",
    "        print(article['_id'], article['text'], '', sep='\\n', file=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
